(*Definitions*)X={{0,0},{0,1},{1,0},{1,1}};y={0,1,1,0};DMax[x_]:=Piecewise[{{0,x==0},{1,x>0},{0,x<0}}];(*Inputs*)W={{0.1,0.2},{0.3,0.4}};c={0.1,0.2};w={0.5,-0.6};b=-0.3;a=0.01;(*LearningRate*)(*InitialPrediction*)MC=ConstantArray[c,Dimensions[X.W][[1]]];Mb=ConstantArray[b,Dimensions[X][[1]]];A1=X.W+MC;H1=Map[Max[0,#]&,A1,{2}];yh=H1.w+Mb;Print["Initial Prediction: ",yh]Print["Initial Error: ",Transpose[yh-y].(yh-y)];
Initial Prediction: {-0.37,-0.46,-0.44,-0.53}
Initial Error: 4.623
(*GradientDescentAlgorithm*)For[i=1,i≤1000,i++,dJdb=Transpose[2(yh-y)].ConstantArray[1,Length[yh-y]];dJdw=2*Transpose[H1].(yh-y);dJdc=Transpose[2*Outer[Times,yh-y,w]*Map[DMax,A1,{2}]].ConstantArray[1,Length[y]];dJdW=Transpose[X].(2(Outer[Times,yh-y,w]*Map[DMax,A1,{2}]));b-=a*dJdb;w-=a*dJdw;c-=a*dJdc;W-=a*dJdW;MC=ConstantArray[c,Dimensions[X.W][[1]]];Mb=ConstantArray[b,Dimensions[X][[1]]];A1=X.W+MC;H1=Map[Max[0,#]&,A1,{2}];yh=H1.w+Mb;](*Results*)Print["Final Error: ",Transpose[yh-y].(yh-y)];Print["Weight b: ",b];Print["Weight w: ",w];Print["Weight c: ",c];Print["Weight W: ",W];Print["Final Prediction: ",yh]
Final Error: 0.000100299
Weight b: -0.113902
Weight w: {1.21307,-1.86435}
Weight c: {0.0998303,-1.06259}
Weight W: {{0.814761,1.06265},{0.814765,1.06266}}
Final Prediction: {0.00719933,0.995448,0.99545,0.00265486}